/*
 * Licensed to the Apache Software Foundation (ASF) under one or more
 * contributor license agreements.  See the NOTICE file distributed with
 * this work for additional information regarding copyright ownership.
 * The ASF licenses this file to You under the Apache License, Version 2.0
 * (the "License"); you may not use this file except in compliance with
 * the License.  You may obtain a copy of the License at
 *
 *    http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */
package com.zt.bigdata.spark.spark.sql

import org.apache.spark.sql.types.{StringType, StructField, StructType}
import org.apache.spark.sql.{Row, SparkSession}

object StudentAnalysis {

  case class Person(name: String, age: Long)

  def main(args: Array[String]) {
    val spark = SparkSession
      .builder()
      .master("local")
      .appName("Spark SQL data sources example")
      .config("spark.some.config.option", "some-value")
      .getOrCreate()

    //https://blog.csdn.net/feloxx/article/details/72819964
    //https://www.cnblogs.com/canyangfeixue/p/9630911.html
    //todo learn the blog
    runData(spark)

    spark.stop()
  }

  private def runData(spark: SparkSession): Unit = {
    val accessRDD = spark.sparkContext.textFile("src/main/resources/access.log", 3)
    import spark.implicits._


    val schemaString = new StringBuilder
    for (i <- 0 until 12) {
      schemaString.append("field").append(i + 1).append(" ")
    }

    // Generate the schema based on the string of schema
    val fields = schemaString.toString().split(" ")
      .map(fieldName => StructField(fieldName, StringType, nullable = true))
    val schema = StructType(fields)

    val rowRDD = accessRDD
      .map(x => x.split("\\|"))
      .map(x => Row(x: _*))

    val accessDF = spark.createDataFrame(rowRDD, schema)
    accessDF.createTempView("accessLog")

    println(accessDF.schema)
    import org.apache.spark.sql.functions._
    accessDF.groupBy("field1")
      .agg(count("field1").alias("cnt"))
      .select("field1", "cnt").show()

    spark.sql("select field1, count(*) as cnt from accessLog group by field1").show()


    accessDF.groupBy(substring($"field1", 0, 16).alias("field"))
      .agg(count("*").alias("cnt"))
      .select("field", "cnt").show()

    spark.sql(
      """
        |select substring(field1,0, 16) as field ,count(*) as cnt from accessLog group by substring(field1,0, 16)
      """.stripMargin).show()

  }
}
